# -*- coding: utf-8 -*-
# @Time : 2022/5/25 15:26
# @FileName: FundCompany.py
# @Mail: lizhigen1996@aliyun.com

__author__ = 'Zhigen.li'

import os
import re
import copy
import requests
from enum import Enum
from lxml import etree
from datetime import datetime
from multiprocessing import Pool
from urllib.request import urlcleanup

class CompanyParseEnum(Enum):
    INFO = 1
    FUNDSCALE = 2
    FUNDLIST = 3
    STOCK10 = 4
    CATEGORY = 5

FundCompanyName = {
    CompanyParseEnum.INFO: '基金公司基本信息',
    CompanyParseEnum.FUNDSCALE: '基金公司股票型和混合型规模、数量、经理数量排名',
    CompanyParseEnum.FUNDLIST: '基金公司下的基金清单',
    CompanyParseEnum.STOCK10: '基金公司的10大持仓股票',
    CompanyParseEnum.CATEGORY: '基金公司下的行业配置',
}

PathPrefix = 'C:\\Config\\'

# 基金公司情况
class FundCompany(object):
    """
        2.1基金公司基本信息  need_company_info
        2.2基金公司股票型和混合型规模、数量、经理数量排名  need_company_fundscale
        2.3基金公司下的基金清单  need_company_fundlist
        2.4公司的10大持仓股票  need_company_10stock
        2.5公司下的行业配置 need_industry_category
    """
    def __init__(self):
        self.start_url = 'http://fund.eastmoney.com/Company/default.html'
        self.str_now_day = datetime.now().strftime('%Y%m%d')
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36',
            'Referer': 'http://fund.eastmoney.com/'
        }

    def parse(self, parseType:CompanyParseEnum, processes=2):
        self.parseType = parseType
        self.str_now_day = datetime.now().strftime('%Y%m%d')
        html = requests.get(self.start_url, headers=self.headers)
        html.encoding = 'utf-8'
        html = html.text
        # html = urlopen(Request(self.start_url, headers=self.headers)).read().decode('utf8')
        response = etree.HTML(html)
        a_list = response.xpath("//div[@class='sencond-block']/a")

        itemList = []
        for a in a_list:
            item = {"company_shortName": a.xpath("./text()")}
            if len(item["company_shortName"]) == 0:
                item["company_shortName"] = None
                continue
            item["company_shortName"] = item["company_shortName"][0]
            item["company_url"] = "http://fund.eastmoney.com" + a.xpath("./@href")[0]
            item["company_code"] = re.findall(r'Company/(.*?)\.html', item["company_url"])[0]
            itemList.append(item)

        pool = Pool(processes=processes)
        output = pool.map(self.parse_pool_func, itemList)

        path = PathPrefix + "{}_{}.csv".format(FundCompanyName[self.parseType], self.str_now_day)
        if os.path.exists(path):
            os.remove(path)
        self.save_item_in_csv(output, path)

        urlcleanup()  # 清除缓存

    def parse_pool_func(self, item):
        if self.parseType == CompanyParseEnum.INFO:
            url = "http://fund.eastmoney.com/Company/{}.html".format(item["company_code"])
            it = self.parse_company_info(url, copy.deepcopy(item))
            return it
        elif self.parseType == CompanyParseEnum.FUNDSCALE:
            items = []
            for fundType in ["25", "27"]:  # funyType=25:股票型 27:混合型
                url = "http://fund.eastmoney.com/Company/home/Gmtable?gsId={}&fundType={}".format(item["company_code"], fundType)
                it = self.parse_company_fundscale(url, copy.deepcopy(item))
                items.append(it)
            return items
        elif self.parseType == CompanyParseEnum.FUNDLIST:
            items = []
            for funyType in ["001", "002"]:  # funyType=001是股票型 002是混合型
                url = "http://fund.eastmoney.com/Company/home/KFSFundNet?gsid={}&fundType={}".format(item["company_code"], funyType)
                it = self.parse_company_fundList(url, copy.deepcopy(item))
                items.append(it)
            return items
        elif self.parseType == CompanyParseEnum.STOCK10:
            url = "http://fund.eastmoney.com/Company/f10/gscc_{}.html".format(item["company_code"])
            items = self.parse_company_10stock(url, copy.deepcopy(item))
            return items
        elif self.parseType == CompanyParseEnum.CATEGORY:
            url = "http://fund.eastmoney.com/Company/f10/hypz_{}.html".format(item["company_code"])
            items = self.parse_company_industry_category(url, copy.deepcopy(item))
            return items


    def parse_company_info(self, url, item):
        try:
            html = requests.get(url, headers=self.headers)
            html.encoding = 'utf-8'
            html = html.text
            # html = urlopen(Request(url, headers=self.headers)).read().decode('utf8')
            company_code = item.get("company_code")
            company_shortName = item.get("company_shortName")
            response = etree.HTML(html)
            item = {"基金公司代码": company_code,
                    "基金公司": company_shortName,
                    "基金公司全称": response.xpath("//p[@class='ttjj-panel-main-title']/text()"),
                    "位置": response.xpath("//div[@class='firm-contact clearfix']/div[1]/p[1]/label/text()"),
                    "总经理": response.xpath("//div[@class='firm-contact clearfix']/div[1]/p[2]/label/text()"),
                    "官网": response.xpath("//div[@class='firm-contact clearfix']/div[2]/p[1]/label/text()"),
                    "客服热线": response.xpath("//div[@class='firm-contact clearfix']/div[2]/p[2]/label/text()"),
                    "管理规模": response.xpath("//a[text()='管理规模']/../label/text()"),
                    "基金数量": response.xpath("//div[@class='fund-info']/ul/li[2]/label/a/text()"),
                    "经理人数": response.xpath("//div[@class='fund-info']/ul/li[3]/label/a/text()"),
                    "成立日期": response.xpath("//div[@class='fund-info']/ul/li[5]/label/text()"),
                    "公司性质": response.xpath("//div[@class='fund-info']/ul/li[6]/label/text()")[0].strip()}
            for i_name in ['基金公司全称', '位置', '总经理', '官网', '客服热线',
                           '管理规模', '基金数量', '经理人数', '成立日期']:
                item[i_name] = item[i_name][0] if len(item[i_name]) > 0 else None
            print(item)
            return item
        except Exception as e:
            print(e)
            return None

    # 2.2基金公司股票型和混合型规模、数量、经理数量排名
    def parse_company_fundscale(self, url, item):
        try:
            html = requests.get(url, headers=self.headers)
            html.encoding = 'utf-8'
            html = html.text
            company_code = item.get("company_code")
            company_shortName = item.get("company_shortName")
            response = etree.HTML(html)
            item = {"基金公司代码": company_code,
                    "基金公司": company_shortName,
                    "基金类型": response.xpath("//tr[1]/th[2]/span/text()"),
                    "基金规模（亿元）": response.xpath("//tr[2]/td[2]/text()"),
                    "平均每家公司基金规模": response.xpath("//tr[2]/td[3]/text()"),
                    "基金规模同类排名": response.xpath("//tr[2]/td[4]/text()"),
                    "基金数量": response.xpath("//tr[3]/td[2]/text()"),
                    "平均每家公司基金数量": response.xpath("//tr[3]/td[3]/text()"),
                    "基金数量同类排名": response.xpath("//tr[3]/td[4]/text()"),
                    "基金经理数量": response.xpath("//tr[4]/td[2]/text()"),
                    "平均每家公司基金经理数量": response.xpath("//tr[4]/td[3]/text()"),
                    "基金经理数量同类排名": response.xpath("//tr[4]/td[4]/text()")}
            for i_name in ['基金类型', '基金规模（亿元）', '平均每家公司基金规模', '基金规模同类排名', '基金数量',
                           '平均每家公司基金数量', '基金数量同类排名', '基金经理数量', '平均每家公司基金经理数量',
                           '基金经理数量同类排名']:
                item[i_name] = item[i_name][0] if len(item[i_name]) > 0 else None
            print(item)
            return item
        except Exception as e:
            print(e)
            return None

    # 2.3基金公司下的基金清单
    def parse_company_fundList(self, url, item):
        items = []
        try:
            html = requests.get(url, headers=self.headers)
            html.encoding = 'utf-8'
            html = html.text
            company_code = item.get("company_code")
            company_shortName = item.get("company_shortName")
            response = etree.HTML(html)
            tr_list = response.xpath("//tbody/tr")
            for tr in tr_list:
                item = {"基金公司代码": company_code,
                        "基金公司": company_shortName,
                        "基金名称": tr.xpath("./td/a[1]/text()"),
                        "基金代码": tr.xpath("./td/a[2]/text()")}
                item["基金名称"] = item["基金名称"][0] if len(item["基金名称"]) > 0 else None
                item["基金代码"] = "\t" + tr.xpath("./td/a[2]/text()")[0] if len(item["基金代码"]) > 0 else None
                items.append(item)
                print(item)

            return items
        except Exception as e:
            print(e)
            return items if len(items) > 0 else None

    # 2.4公司的10大持仓股票
    def parse_company_10stock(self, url, item):
        items = []
        try:
            html = requests.get(url, headers=self.headers)
            html.encoding = 'utf-8'
            html = html.text
            company_code = item.get("company_code")
            company_shortName = item.get("company_shortName")
            response = etree.HTML(html)
            tr_list = response.xpath("//table[@class='ttjj-table ttjj-table-hover']/tbody[1]/tr")
            for tr in tr_list:
                item = dict()
                item["基金公司代码"] = company_code
                item["基金公司"] = company_shortName
                item["股票代码"] = tr.xpath("./td[2]/a/text()")[0]
                item["股票代码"] = "\t" + item["股票代码"]
                item["股票名称"] = tr.xpath("./td[3]/a/text()")[0]
                item["配置的基金数"] = tr.xpath("./td[5]/a/text()")[0]  # 本公司持有基金数
                item["配置的基金占比"] = tr.xpath("./td[6]/text()")[0]  # 占总净值比例
                item["配置数量（万股）"] = tr.xpath("./td[7]/text()")[0]  # 持股数(万股)
                item["配置市值（万元）"] = tr.xpath("./td[8]/text()")[0]  # 持仓市值(万元)
                items.append(item)
                print(item)
            return items
        except Exception as e:
            print(e)
            return items if len(items) > 0 else None

    # 2.5公司下的行业配置
    def parse_company_industry_category(self, url, item):
        items = []
        try:
            html = requests.get(url, headers=self.headers)
            html.encoding = 'utf-8'
            html = html.text
            company_code = item.get("company_code")
            company_shortName = item.get("company_shortName")
            response = etree.HTML(html)
            tr_list = response.xpath("//table[@class='ttjj-table ttjj-table-hover']//tr")[1:]  # [1:]去标题
            for tr in tr_list:
                item = {"基金公司代码": company_code,
                        "基金公司": company_shortName,
                        "行业": tr.xpath("./td[2]/text()")[0],
                        "配置的基金数": tr.xpath("./td[4]/a/text()")[0],
                        "配置的基金占比": tr.xpath("./td[5]/text()")[0],
                        "配置金额（万元）": tr.xpath("./td[6]/text()")[0]}
                items.append(item)
                print(item)
            return items
        except Exception as e:
            print(e)
            return items if len(items) > 0 else None

    def save_item_in_csv(self, items: list, file_name):
        with open(file_name, "a", encoding="utf-8-sig", newline='') as fp:
            if self.parseType == CompanyParseEnum.INFO:
                fp.write(','.join(items[0].keys()) + '\n')
                for item in items:
                    if item == None:
                        continue
                    values = list(item.values())
                    for idx, v in enumerate(values):
                        if v == None:
                            values[idx] = ''
                        elif ',' in v:
                            values[idx] = v.replace(',', '')
                    fp.write(','.join(values) + '\n')
            elif self.parseType == CompanyParseEnum.FUNDSCALE or self.parseType == CompanyParseEnum.STOCK10 \
                    or self.parseType == CompanyParseEnum.CATEGORY:
                fp.write(','.join(items[0][0].keys()) + '\n')
                for item in items:
                    for it in item:
                        if it == None:
                            continue
                        values = list(it.values())
                        for idx, v in enumerate(values):
                            if v == None:
                                values[idx] = ''
                            elif ',' in v:
                                values[idx] = v.replace(',', '')
                        fp.write(','.join(values) + '\n')
            elif self.parseType == CompanyParseEnum.FUNDLIST:
                fp.write(','.join(items[0][0][0].keys()) + '\n')
                for item in items:
                    for its in item:
                        for it in its:
                            if it == None:
                                continue
                            values = list(it.values())
                            for idx, v in enumerate(values):
                                if v == None:
                                    values[idx] = ''
                                elif ',' in v:
                                    values[idx] = v.replace(',', '')
                            fp.write(','.join(values) + '\n')

if __name__ == '__main__':
    # FundCompanyName = {
    #     CompanyParseEnum.INFO: '基金公司基本信息',
    #     CompanyParseEnum.FUNDSCALE: '基金公司股票型和混合型规模、数量、经理数量排名',
    #     CompanyParseEnum.FUNDLIST: '基金公司下的基金清单',
    #     CompanyParseEnum.STOCK10: '基金公司的10大持仓股票',
    #     CompanyParseEnum.CATEGORY: '基金公司下的行业配置',
    # }
    startTime = datetime.now()
    f = FundCompany()
    f.parse(CompanyParseEnum.INFO)
    f.parse(CompanyParseEnum.FUNDSCALE)
    f.parse(CompanyParseEnum.FUNDLIST)
    f.parse(CompanyParseEnum.STOCK10)
    f.parse(CompanyParseEnum.CATEGORY)
    print(datetime.now() - startTime)

